<!DOCTYPE html>





<html class="theme-next gemini use-motion" lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="generator" content="Hexo 3.9.0">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=7.3.0">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=7.3.0">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=7.3.0">
  <link rel="mask-icon" href="/images/logo.svg?v=7.3.0" color="#222">

<link rel="stylesheet" href="/css/main.css?v=7.3.0">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css?v=4.7.0">
  <link rel="stylesheet" href="/lib/pace/pace-theme-big-counter.min.css?v=1.0.2">
  <script src="/lib/pace/pace.min.js?v=1.0.2"></script>


<script id="hexo-configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Gemini',
    version: '7.3.0',
    exturl: false,
    sidebar: {"position":"right","display":"post","offset":12,"onmobile":false},
    back2top: {"enable":true,"sidebar":true,"scrollpercent":true},
    save_scroll: false,
    copycode: {"enable":true,"show_result":false,"style":null},
    fancybox: false,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    algolia: {
      appID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    path: 'search.xml',
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    translation: {
      copy_button: '复制',
      copy_success: '复制成功',
      copy_failure: '复制失败'
    }
  };
</script>

  <meta name="description" content="Note&amp;#x6BCF;&amp;#x4E2A;&amp;#x5E38;&amp;#x7528;API&amp;#x90FD;&amp;#x7ED3;&amp;#x5408;&amp;#x5B9E;&amp;#x4F8B;&amp;#x89E3;&amp;#x91CA;&amp;#x5E76;&amp;#x4E14;&amp;#x7528;&amp;#x4E86;Java&amp;#x51FD;&amp;#x6570;&amp;#x5F0F;&amp;#x7F16;&amp;#x7A0B;&amp;#x5199;&amp;#x7684;..&amp;#x6211;&amp;#x">
<meta name="keywords" content="spark">
<meta property="og:type" content="article">
<meta property="og:title" content="spark-rdd-JavaAPI手册">
<meta property="og:url" content="http://ithuhui.com/2018/11/10/spark-rdd-common-java-api/index.html">
<meta property="og:site_name" content="ithuhui">
<meta property="og:description" content="Note&amp;#x6BCF;&amp;#x4E2A;&amp;#x5E38;&amp;#x7528;API&amp;#x90FD;&amp;#x7ED3;&amp;#x5408;&amp;#x5B9E;&amp;#x4F8B;&amp;#x89E3;&amp;#x91CA;&amp;#x5E76;&amp;#x4E14;&amp;#x7528;&amp;#x4E86;Java&amp;#x51FD;&amp;#x6570;&amp;#x5F0F;&amp;#x7F16;&amp;#x7A0B;&amp;#x5199;&amp;#x7684;..&amp;#x6211;&amp;#x">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2019-08-20T06:17:15.253Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="spark-rdd-JavaAPI手册">
<meta name="twitter:description" content="Note&amp;#x6BCF;&amp;#x4E2A;&amp;#x5E38;&amp;#x7528;API&amp;#x90FD;&amp;#x7ED3;&amp;#x5408;&amp;#x5B9E;&amp;#x4F8B;&amp;#x89E3;&amp;#x91CA;&amp;#x5E76;&amp;#x4E14;&amp;#x7528;&amp;#x4E86;Java&amp;#x51FD;&amp;#x6570;&amp;#x5F0F;&amp;#x7F16;&amp;#x7A0B;&amp;#x5199;&amp;#x7684;..&amp;#x6211;&amp;#x">
  <link rel="canonical" href="http://ithuhui.com/2018/11/10/spark-rdd-common-java-api/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome: false,
    isPost: true,
    isPage: false,
    isArchive: false
  };
</script>

  <title>spark-rdd-JavaAPI手册 | ithuhui</title>
  


  <script>
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?b7300352bcf49d6bfa3a57b8a0e37cdb";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>






  <noscript>
  <style>
  .use-motion .motion-element,
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-title { opacity: initial; }

  .use-motion .logo,
  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">

  <div class="container sidebar-position-right">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta">

    <div class="custom-logo-site-title">
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">ithuhui</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
        <p class="site-subtitle">记录一切</p>
      
  </div>

  <div class="site-nav-toggle">
    <button aria-label="切换导航栏">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>


<nav class="site-nav">
  
  <ul id="menu" class="menu">
      
      
      
        
        <li class="menu-item menu-item-home">
      
    

    <a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home"></i> <br>首页</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-about">
      
    

    <a href="/about/" rel="section"><i class="menu-item-icon fa fa-fw fa-user"></i> <br>关于</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-tags">
      
    

    <a href="/tags/" rel="section"><i class="menu-item-icon fa fa-fw fa-tags"></i> <br>标签</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-categories">
      
    

    <a href="/categories/" rel="section"><i class="menu-item-icon fa fa-fw fa-th"></i> <br>分类</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-archives">
      
    

    <a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive"></i> <br>归档</a>

  </li>
      <li class="menu-item menu-item-search">
        <a href="javascript:;" class="popup-trigger">
        
          <i class="menu-item-icon fa fa-search fa-fw"></i> <br>搜索</a>
      </li>
    
  </ul>

</nav>
  <div class="site-search">
    
  <div class="popup search-popup">
  <div class="search-header">
    <span class="search-icon">
      <i class="fa fa-search"></i>
    </span>
    <div class="search-input-wrapper">
      <input autocomplete="off" autocorrect="off" autocapitalize="none"
             placeholder="搜索..." spellcheck="false"
             type="text" id="search-input">
    </div>
    <span class="popup-btn-close">
      <i class="fa fa-times-circle"></i>
    </span>
  </div>
  <div id="search-result"></div>
</div>


  </div>
</div>
    </header>

    

  <a href="https://github.com/ithuhui" class="github-corner" title="Follow me on GitHub" aria-label="Follow me on GitHub" rel="noopener" target="_blank"><svg width="80" height="80" viewBox="0 0 250 250" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a>


    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
            

          <div id="content" class="content page-post-detail">
            

  <div id="posts" class="posts-expand">
    

  <article class="post" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://ithuhui.com/2018/11/10/spark-rdd-common-java-api/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="冷大锤`">
      <meta itemprop="description" content="发牌的是上帝，出牌的是自己">
      <meta itemprop="image" content="/images/head.png">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="ithuhui">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">spark-rdd-JavaAPI手册

          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
                
              

              <time title="创建时间：2018-11-10 02:35:19" itemprop="dateCreated datePublished" datetime="2018-11-10T02:35:19+08:00">2018-11-10</time>
            </span>
          
            

            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2019-08-20 14:17:15" itemprop="dateModified" datetime="2019-08-20T14:17:15+08:00">2019-08-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/bigdata/" itemprop="url" rel="index"><span itemprop="name">bigdata</span></a></span>

                
                
              
            </span>
          

          
            <span id="/2018/11/10/spark-rdd-common-java-api/" class="post-meta-item leancloud_visitors" data-flag-title="spark-rdd-JavaAPI手册" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="fa fa-comment-o"></i>
      </span>
        
      
      <span class="post-meta-item-text">Valine：</span>
    
    <a title="valine" href="/2018/11/10/spark-rdd-common-java-api/#comments" itemprop="discussionUrl"><span class="post-comments-count valine-comment-count" data-xid="/2018/11/10/spark-rdd-common-java-api/" itemprop="commentCount"></span></a>
  </span>
  
  
          <br>
            <span class="post-meta-item" title="本文字数">
              <span class="post-meta-item-icon">
                <i class="fa fa-file-word-o"></i>
              </span>
              
                <span class="post-meta-item-text">本文字数：</span>
              
              <span>10k</span>
            </span>
          
            <span class="post-meta-item" title="阅读时长">
              <span class="post-meta-item-icon">
                <i class="fa fa-clock-o"></i>
              </span>
              
                <span class="post-meta-item-text">阅读时长 &asymp;</span>
              
              <span>9 分钟</span>
            </span>
          

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h1 id="Note"><a href="#Note" class="headerlink" title="Note"></a>Note</h1><p>&#x6BCF;&#x4E2A;&#x5E38;&#x7528;API&#x90FD;&#x7ED3;&#x5408;&#x5B9E;&#x4F8B;&#x89E3;&#x91CA;&#x5E76;&#x4E14;&#x7528;&#x4E86;Java&#x51FD;&#x6570;&#x5F0F;&#x7F16;&#x7A0B;&#x5199;&#x7684;..&#x6211;&#x8FD9;&#x61D2;&#x4EBA;&#x591F;&#x826F;&#x5FC3;&#x4E86;&#x2026;.</p><a id="more"></a>
<p>&#x80FD;&#x5E2E;&#x5230;&#x4F60;&#x4EEC;&#x7684;&#x8BDD;&#xFF0C;&#x662F;&#x6211;&#x7684;&#x8363;&#x5E78;</p>
<ul>
<li>wiki&#x89E3;&#x91CA;</li>
</ul>
<p><strong>Spark RDD</strong>&#xFF08;&#x82F1;&#x8BED;&#xFF1A;Resilient Distributed Dataset&#xFF0C;&#x5F39;&#x6027;&#x5206;&#x5E03;&#x5F0F;&#x6570;&#x636E;&#x96C6;&#xFF09;&#x662F;&#x4E00;&#x79CD;&#x6570;&#x636E;&#x5B58;&#x50A8;&#x96C6;&#x5408;&#x3002;&#x53EA;&#x80FD;&#x7531;&#x5B83;&#x652F;&#x6301;&#x7684;&#x6570;&#x636E;&#x6E90;&#x6216;&#x662F;&#x7531;&#x5176;&#x4ED6;RDD&#x7ECF;&#x8FC7;&#x4E00;&#x5B9A;&#x7684;&#x8F6C;&#x6362;&#xFF08;Transformation&#xFF09;&#x6765;&#x4EA7;&#x751F;&#x3002;&#x5728;RDD&#x4E0A;&#x53EF;&#x4EE5;&#x6267;&#x884C;&#x7684;&#x64CD;&#x4F5C;&#x6709;&#x4E24;&#x79CD;&#x8F6C;&#x6362;&#xFF08;Transformation&#xFF09;&#x548C;&#x884C;&#x52A8;&#xFF08;Action&#xFF09;&#xFF0C;&#x6BCF;&#x4E2A; RDD &#x90FD;&#x8BB0;&#x5F55;&#x4E86;&#x81EA;&#x5DF1;&#x662F;&#x5982;&#x4F55;&#x7531;&#x6301;&#x4E45;&#x5316;&#x5B58;&#x50A8;&#x4E2D;&#x7684;&#x6E90;&#x6570;&#x636E;&#x8BA1;&#x7B97;&#x5F97;&#x51FA;&#x7684;&#xFF0C;&#x5373;&#x5176;&#x8840;&#x7EDF;&#xFF08;Lineage&#xFF09;</p>
<ul>
<li>&#x91CD;&#x70B9;</li>
</ul>
<ol>
<li>RDD&#x662F;&#x4E00;&#x4E2A;<strong>&#x5206;&#x5E03;&#x5F0F;&#x96C6;&#x5408;</strong></li>
<li>SparkRDD&#x7279;&#x6027;&#x60F0;&#x6027;&#x884C;&#x4E3A;&#xFF0C;&#x53EA;&#x6709;&#x6FC0;&#x6D3B;Action&#x64CD;&#x4F5C;&#x624D;&#x4F1A;&#x771F;&#x6B63;&#x5F00;&#x59CB;&#x8BA1;&#x7B97;</li>
<li>&#x4E24;&#x79CD;&#x8F6C;&#x6362;&#x64CD;&#x4F5C;Transformation&#xFF08;&#x60F0;&#x6027;&#xFF0C;&#x53EA;&#x4EE3;&#x8868;&#x903B;&#x8F91;&#xFF09;&#x548C;&#x884C;&#x52A8;Action(&#x8D77;&#x5E8A;&#xFF0C;&#x5F00;&#x59CB;&#x6267;&#x884C;)</li>
<li>&#x8840;&#x7EDF;&#xFF08;Lineage&#xFF09;&#x600E;&#x4E48;&#x7406;&#x89E3;&#x5462;&#xFF1F;&#x8840;&#x7EDF;&#x5177;&#x6709;&#x7EE7;&#x627F;&#x7684;&#x7279;&#x6027;&#xFF0C;&#x80FD;&#x8FFD;&#x6839;&#x6EAF;&#x6E90;&#x3002;Lineage&#x8BB0;&#x5F55;&#x4E86;&#x4F60;&#x7684;Transformation&#x548C;Action&#xFF0C;&#x901A;&#x8FC7; Lineage &#x83B7;&#x53D6;&#x8DB3;&#x591F;&#x7684;&#x4FE1;&#x606F;&#x6765;&#x91CD;&#x65B0;&#x8FD0;&#x7B97;&#x548C;&#x6062;&#x590D;&#x4E22;&#x5931;&#x7684;&#x6570;&#x636E;&#x5206;&#x533A;&#xFF0C;&#x8FD9;&#x4E5F;&#x662F;&#x5B83;&#x7684;&#x5BB9;&#x9519;&#x539F;&#x7406;&#xFF0C;&#x67D0;&#x8282;&#x70B9;&#x6302;&#x6389;&#xFF0C;&#x518D;&#x901A;&#x8FC7;Lineage&#x8BB0;&#x5F55;&#x91CD;&#x65B0;&#x6267;&#x884C;&#x4E00;&#x6B21;&#x64CD;&#x4F5C;&#x5373;&#x53EF;</li>
</ol>
<ul>
<li>&#x5EFA;&#x8BAE;</li>
</ul>
<ol>
<li>&#x51FD;&#x6570;&#x5F0F;&#x7F16;&#x7A0B;&#x4E86;&#x89E3;&#x4E0B;</li>
<li>&#x5199;java&#x7684;&#xFF0C;&#x53EF;&#x4EE5;&#x5148;Java&#x5165;&#x624B;&#xFF0C;&#x4F46;&#x662F;&#x4E0D;&#x80FD;&#x4E22;&#x5F03;scala&#x7248;&#x672C;&#xFF0C;spark&#x7684;&#x7238;&#x7238;&#x6BD5;&#x7ADF;&#x662F;scala.. &#xFF08;scala&#x4E5F;&#x66F4;&#x5BB9;&#x6613;&#xFF09;</li>
<li>&#x4EE3;&#x7801;&#x6CE8;&#x91CA;&#x8BF7;&#x67E5;&#x6536;</li>
</ol>
<ul>
<li><p>&#x5E38;&#x7528;RDD&#xFF08;&#x6240;&#x6709;&#x65B9;&#x6CD5;&#x6211;&#x90FD;&#x5199;&#x4E86;DEMO&#xFF09;</p>
<p>Transformation&#x884C;&#x4E3A;&#xFF1A;</p>
<table>
<thead>
<tr>
<th align="left">&#x65B9;&#x6CD5;</th>
<th>&#x5B98;&#x65B9;&#x89E3;&#x91CA;</th>
<th><strong>Hui&#x89E3;&#x91CA;</strong></th>
</tr>
</thead>
<tbody><tr>
<td align="left"><strong>map(func)</strong></td>
<td><strong>&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x65B0;&#x7684;&#x5206;&#x5E03;&#x5F0F;&#x6570;&#x636E;&#x96C6;&#xFF0C;&#x8BE5;&#x6570;&#x636E;&#x96C6;&#x7531;&#x6BCF;&#x4E00;&#x4E2A;&#x8F93;&#x5165;&#x5143;&#x7D20;&#x7ECF;&#x8FC7;func&#x51FD;&#x6570;&#x8F6C;&#x6362;&#x540E;&#x7EC4;&#x6210;</strong></td>
<td><strong>&#x5143;&#x7D20;&#x8F6C;&#x6362; . &#x53C2;&#x6570;-&gt;&#x4F60;&#x5E0C;&#x671B;&#x8981;&#x7684;&#x53C2;&#x6570;</strong></td>
</tr>
<tr>
<td align="left"><strong>fitler(func)</strong></td>
<td><strong>&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x65B0;&#x7684;&#x6570;&#x636E;&#x96C6;&#xFF0C;&#x8BE5;&#x6570;&#x636E;&#x96C6;&#x7531;&#x7ECF;&#x8FC7;func&#x51FD;&#x6570;&#x8BA1;&#x7B97;&#x540E;&#x8FD4;&#x56DE;&#x503C;&#x4E3A;true&#x7684;&#x8F93;&#x5165;&#x5143;&#x7D20;&#x7EC4;&#x6210;</strong></td>
<td><strong>&#x5143;&#x7D20;&#x8F6C;&#x6362;. &#x53C2;&#x6570;-&gt;&#x6570;&#x7EC4;&#x53C2;&#x6570;</strong></td>
</tr>
<tr>
<td align="left"><strong>flatMap(func)</strong></td>
<td><strong>&#x7C7B;&#x4F3C;&#x4E8E;map&#xFF0C;&#x4F46;&#x662F;&#x6BCF;&#x4E00;&#x4E2A;&#x8F93;&#x5165;&#x5143;&#x7D20;&#x53EF;&#x4EE5;&#x88AB;&#x6620;&#x5C04;&#x4E3A;0&#x6216;&#x591A;&#x4E2A;&#x8F93;&#x51FA;&#x5143;&#x7D20;&#xFF08;&#x56E0;&#x6B64;func&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x5E8F;&#x5217;&#xFF0C;&#x800C;&#x4E0D;&#x662F;&#x5355;&#x4E00;&#x5143;&#x7D20;&#xFF09;</strong></td>
<td><strong>&#x5143;&#x7D20;&#x8F6C;&#x6362;. &#x53C2;&#x6570;-&gt;pair&#x53C2;&#x6570; (key value)</strong></td>
</tr>
<tr>
<td align="left"><strong>mapPartitions(func)</strong></td>
<td><strong>&#x7C7B;&#x4F3C;&#x4E8E;map&#xFF0C;&#x4F46;&#x72EC;&#x7ACB;&#x5730;&#x5728;RDD&#x4E0A;&#x6BCF;&#x4E00;&#x4E2A;&#x5206;&#x7247;&#x4E0A;&#x8FD0;&#x884C;&#xFF0C;&#x56E0;&#x6B64;&#x5728;&#x7C7B;&#x578B;&#x4E3A;T&#x7684;RDD&#x4E0A;&#x8FD0;&#x884C;&#x65F6;&#xFF0C;func&#x51FD;&#x6570;&#x7C7B;&#x578B;&#x5FC5;&#x987B;&#x662F;Iterator[T]=&gt;Iterator[U]</strong></td>
<td><strong>&#x5143;&#x7D20;&#x8F6C;&#x6362;,&#x5728;&#x6BCF;&#x4E00;&#x4E2A;&#x5206;&#x533A;&#x5185;&#x90E8;&#x8FDB;&#x884C;&#x5143;&#x7D20;&#x8F6C;&#x6362;.</strong></td>
</tr>
<tr>
<td align="left"><strong>mapPartitionsWithIndex(func)</strong></td>
<td><strong>&#x7C7B;&#x4F3C;&#x4E8E;mapPartitons&#xFF0C;&#x4F46;func&#x5E26;&#x6709;&#x4E00;&#x4E2A;&#x6574;&#x6570;&#x53C2;&#x6570;&#x8868;&#x793A;&#x5206;&#x7247;&#x7684;&#x7D22;&#x5F15;&#x503C;&#x3002;&#x56E0;&#x6B64;&#x5728;&#x7C7B;&#x578B;&#x4E3A;T&#x7684;RDD&#x4E0A;&#x8FD0;&#x884C;&#x65F6;&#xFF0C;func&#x51FD;&#x6570;&#x7C7B;&#x578B;&#x5FC5;&#x987B;&#x662F;(Int,Iterator[T])=&gt;Iterator[U]</strong></td>
<td><strong>&#x5143;&#x7D20;&#x8F6C;&#x6362;,&#x5728;&#x6BCF;&#x4E00;&#x4E2A;&#x5206;&#x533A;&#x5185;&#x90E8;&#x8FDB;&#x884C;&#x5143;&#x7D20;&#x8F6C;&#x6362;.</strong></td>
</tr>
<tr>
<td align="left"><strong>sample(withReplacement,fraction,seed)</strong></td>
<td><strong>&#x6839;&#x636E;fraction&#x6307;&#x5B9A;&#x7684;&#x6BD4;&#x4F8B;&#x5BF9;&#x6570;&#x636E;&#x8FDB;&#x884C;&#x91C7;&#x6837;&#xFF0C;&#x53EF;&#x4EE5;&#x9009;&#x62E9;&#x662F;&#x5426;&#x7528;&#x968F;&#x673A;&#x6570;&#x8FDB;&#x884C;&#x66FF;&#x6362;&#xFF0C;seed&#x7528;&#x4E8E;&#x968F;&#x673A;&#x6570;&#x751F;&#x6210;&#x5668;&#x79CD;&#x5B50;</strong></td>
<td><strong>&#x91C7;&#x6837;</strong></td>
</tr>
<tr>
<td align="left"><strong>union(otherDataSet)</strong></td>
<td><strong>&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x65B0;&#x6570;&#x636E;&#x96C6;&#xFF0C;&#x65B0;&#x6570;&#x636E;&#x96C6;&#x662F;&#x7531;&#x539F;&#x6570;&#x636E;&#x96C6;&#x548C;&#x53C2;&#x6570;&#x6570;&#x636E;&#x96C6;&#x8054;&#x5408;&#x800C;&#x6210;</strong></td>
<td><strong>&#x5E76;&#x96C6;</strong></td>
</tr>
<tr>
<td align="left"><strong>distinct([numTasks])</strong></td>
<td><strong>&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x5305;&#x542B;&#x539F;&#x6570;&#x636E;&#x96C6;&#x4E2D;&#x6240;&#x6709;&#x4E0D;&#x91CD;&#x590D;&#x5143;&#x7D20;&#x7684;&#x65B0;&#x6570;&#x636E;&#x96C6;</strong></td>
<td><strong>&#x53BB;&#x91CD;</strong></td>
</tr>
<tr>
<td align="left"><strong>groupByKey([numTasks])</strong></td>
<td><strong>&#x5728;&#x4E00;&#x4E2A;(K,V)&#x6570;&#x636E;&#x96C6;&#x4E0A;&#x8C03;&#x7528;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;(K,Seq[V])&#x5BF9;&#x7684;&#x6570;&#x636E;&#x96C6;&#x3002;&#x6CE8;&#x610F;&#x9ED8;&#x8BA4;&#x60C5;&#x51B5;&#x4E0B;&#xFF0C;&#x53EA;&#x6709;8&#x4E2A;&#x5E76;&#x884C;&#x4EFB;&#x52A1;&#x6765;&#x64CD;&#x4F5C;&#xFF0C;&#x4F46;&#x662F;&#x53EF;&#x4EE5;&#x4F20;&#x5165;&#x4E00;&#x4E2A;&#x53EF;&#x9009;&#x7684;numTasks&#x53C2;&#x6570;&#x6765;&#x6539;&#x53D8;&#x5B83;</strong></td>
<td><strong>&#x5206;&#x7EC4;</strong></td>
</tr>
<tr>
<td align="left"><strong>reduceByKey(func,[numTasks])</strong></td>
<td><strong>&#x5728;&#x4E00;&#x4E2A;(K,V)&#x5BF9;&#x7684;&#x6570;&#x636E;&#x96C6;&#x4E0A;&#x8C03;&#x7528;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;(K,V)&#x5BF9;&#x7684;&#x6570;&#x636E;&#x96C6;&#xFF0C;&#x4F7F;&#x7528;&#x6307;&#x5B9A;&#x7684;reduce&#x51FD;&#x6570;&#xFF0C;&#x5C06;&#x76F8;&#x540C;&#x7684;key&#x7684;&#x503C;&#x805A;&#x5408;&#x5230;&#x4E00;&#x8D77;&#x3002;&#x4E0E;groupByKey&#x7C7B;&#x4F3C;&#xFF0C;reduceByKey&#x4EFB;&#x52A1;&#x7684;&#x4E2A;&#x6570;&#x662F;&#x53EF;&#x4EE5;&#x901A;&#x8FC7;&#x7B2C;&#x4E8C;&#x4E2A;&#x53EF;&#x9009;&#x53C2;&#x6570;&#x6765;&#x8BBE;&#x7F6E;&#x7684;</strong></td>
<td><strong>&#x805A;&#x5408;</strong></td>
</tr>
<tr>
<td align="left"><strong>sortByKey([[ascending],numTasks])</strong></td>
<td><strong>&#x5728;&#x4E00;&#x4E2A;(K,V)&#x5BF9;&#x7684;&#x6570;&#x636E;&#x96C6;&#x4E0A;&#x8C03;&#x7528;&#xFF0C;K&#x5FC5;&#x987B;&#x5B9E;&#x73B0;Ordered&#x63A5;&#x53E3;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x6309;&#x7167;Key&#x8FDB;&#x884C;&#x6392;&#x5E8F;&#x7684;(K,V)&#x5BF9;&#x6570;&#x636E;&#x96C6;&#x3002;&#x5347;&#x5E8F;&#x6216;&#x964D;&#x5E8F;&#x7531;ascending&#x5E03;&#x5C14;&#x53C2;&#x6570;&#x51B3;&#x5B9A;</strong></td>
<td><strong>&#x6392;&#x5E8F;</strong></td>
</tr>
<tr>
<td align="left"><strong>join(otherDataset0,[numTasks])</strong></td>
<td><strong>&#x5728;&#x7C7B;&#x578B;&#x4E3A;(K,V)&#x548C;(K,W)&#x6570;&#x636E;&#x96C6;&#x4E0A;&#x8C03;&#x7528;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x76F8;&#x540C;&#x7684;key&#x5BF9;&#x5E94;&#x7684;&#x6240;&#x6709;&#x5143;&#x7D20;&#x5728;&#x4E00;&#x8D77;&#x7684;(K,(V,W))&#x6570;&#x636E;&#x96C6;</strong></td>
<td><strong>&#x96C6;&#x5408;&#x5173;&#x8054;. &#x5408;&#x5E76;&#x76F8;&#x540C;key&#x7684;value</strong></td>
</tr>
<tr>
<td align="left"><strong>cogroup(otherDataset,[numTasks])</strong></td>
<td><strong>&#x5728;&#x7C7B;&#x578B;&#x4E3A;(K,V)&#x548C;(K,W)&#x6570;&#x636E;&#x96C6;&#x4E0A;&#x8C03;&#x7528;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;(K,Seq[V],Seq[W])&#x5143;&#x7956;&#x7684;&#x6570;&#x636E;&#x96C6;&#x3002;&#x8FD9;&#x4E2A;&#x64CD;&#x4F5C;&#x4E5F;&#x53EF;&#x4EE5;&#x79F0;&#x4E3A;groupwith</strong></td>
<td><strong>&#x5206;&#x7EC4;</strong></td>
</tr>
<tr>
<td align="left"><strong>cartesain(ohterDataset)</strong></td>
<td><strong>&#x7B1B;&#x5361;&#x5C14;&#x79EF;&#xFF0C;&#x5728;&#x7C7B;&#x578B;&#x4E3A;T&#x548C;U&#x7C7B;&#x578B;&#x7684;&#x6570;&#x636E;&#x96C6;&#x4E0A;&#x8C03;&#x7528;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;(T,U)&#x5BF9;&#x6570;&#x636E;&#x96C6;(&#x4E24;&#x4E24;&#x7684;&#x5143;&#x7D20;&#x5BF9;)</strong></td>
<td><strong>&#x7B1B;&#x5361;&#x5C14;&#x79EF;</strong></td>
</tr>
</tbody></table>
<p>Action&#x884C;&#x4E3A;&#xFF1A;</p>
<table>
<thead>
<tr>
<th>&#x65B9;&#x6CD5;</th>
<th></th>
<th>Hui&#x89E3;&#x91CA;</th>
</tr>
</thead>
<tbody><tr>
<td><strong>reduce(func)</strong></td>
<td><strong>&#x901A;&#x8FC7;&#x51FD;&#x6570;func(&#x63A5;&#x6536;&#x4E24;&#x4E2A;&#x53C2;&#x6570;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x53C2;&#x6570;)&#x805A;&#x96C6;&#x6570;&#x636E;&#x96C6;&#x4E2D;&#x7684;&#x6240;&#x6709;&#x5143;&#x7D20;&#x3002;&#x8FD9;&#x4E2A;&#x529F;&#x80FD;&#x5FC5;&#x987B;&#x53EF;&#x4EA4;&#x6362;&#x4E14;&#x53EF;&#x5173;&#x8054;&#x7684;&#xFF0C;&#x4ECE;&#x800C;&#x53EF;&#x4EE5;&#x6B63;&#x786E;&#x7684;&#x5E76;&#x884C;&#x8FD0;&#x884C;</strong></td>
<td><strong>&#x805A;&#x5408;</strong></td>
</tr>
<tr>
<td><strong>collect()</strong></td>
<td><strong>&#x5728;&#x9A71;&#x52A8;&#x7A0B;&#x5E8F;&#x4E2D;&#xFF0C;&#x4EE5;&#x6570;&#x7EC4;&#x5F62;&#x5F0F;&#x8FD4;&#x56DE;&#x6570;&#x636E;&#x96C6;&#x4E2D;&#x7684;&#x6240;&#x6709;&#x5143;&#x7D20;&#x3002;&#x901A;&#x5E38;&#x5728;&#x4F7F;&#x7528;filter&#x6216;&#x8005;&#x5176;&#x4ED6;&#x64CD;&#x4F5C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x8DB3;&#x591F;&#x5C0F;&#x7684;&#x6570;&#x636E;&#x5B50;&#x96C6;&#x540E;&#x518D;&#x4F7F;&#x7528;&#x4F1A;&#x6BD4;&#x8F83;&#x6709;&#x7528;</strong></td>
<td><strong>&#x67E5;&#x8BE2;&#x5168;&#x90E8;</strong></td>
</tr>
<tr>
<td><strong>count()</strong></td>
<td><strong>&#x8FD4;&#x56DE;&#x6570;&#x636E;&#x96C6;&#x5143;&#x7D20;&#x4E2A;&#x6570;</strong></td>
<td><strong>&#x6570;&#x91CF;</strong></td>
</tr>
<tr>
<td><strong>first()</strong></td>
<td><strong>&#x8FD4;&#x56DE;&#x6570;&#x636E;&#x96C6;&#x7B2C;&#x4E00;&#x4E2A;&#x5143;&#x7D20;(&#x7C7B;&#x4F3C;&#x4E8E;take(1))</strong></td>
<td><strong>&#x7B2C;&#x4E00;&#x4E2A;&#x5143;&#x7D20;</strong></td>
</tr>
<tr>
<td><strong>take(n)</strong></td>
<td><strong>&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x7531;&#x6570;&#x636E;&#x96C6;&#x524D;n&#x4E2A;&#x5143;&#x7D20;&#x7EC4;&#x6210;&#x7684;&#x6570;&#x7EC4;</strong></td>
<td><strong>&#x524D;&#x9762;N&#x4E2A;&#x5143;&#x7D20;</strong></td>
</tr>
<tr>
<td><strong>sample(withReplacement,num,seed)</strong></td>
<td><strong>&#x6CE8;&#x610F; &#x8FD9;&#x4E2A;&#x64CD;&#x4F5C;&#x76EE;&#x524D;&#x5E76;&#x975E;&#x5E76;&#x884C;&#x6267;&#x884C;&#xFF0C;&#x800C;&#x662F;&#x7531;&#x9A71;&#x52A8;&#x7A0B;&#x5E8F;&#x8BA1;&#x7B97;&#x6240;&#x6709;&#x7684;&#x5143;&#x7D20;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x6570;&#x7EC4;&#xFF0C;&#x8BE5;&#x6570;&#x7EC4;&#x7531;&#x4ECE;&#x6570;&#x636E;&#x96C6;&#x4E2D;&#x968F;&#x673A;&#x91C7;&#x6837;&#x7684;num&#x4E2A;&#x5143;&#x7D20;&#x7EC4;&#x6210;&#xFF0C;&#x53EF;&#x4EE5;&#x9009;&#x62E9;&#x662F;&#x5426;&#x7531;&#x968F;&#x673A;&#x6570;&#x66FF;&#x6362;&#x4E0D;&#x8DB3;&#x7684;&#x90E8;&#x5206;&#xFF0C;seed&#x7528;&#x6237;&#x6307;&#x5B9A;&#x968F;&#x673A;&#x6570;&#x751F;&#x6210;&#x5668;&#x79CD;&#x5B50;</strong></td>
<td><strong>&#x91C7;&#x6837;</strong></td>
</tr>
<tr>
<td><strong>saveAsTextFile(path)</strong></td>
<td><strong>&#x5C06;&#x6570;&#x636E;&#x96C6;&#x7684;&#x5143;&#x7D20;&#x4EE5;textfile&#x7684;&#x5F62;&#x5F0F;&#x4FDD;&#x5B58;&#x5230;&#x672C;&#x5730;&#x6587;&#x4EF6;&#x7CFB;&#x7EDF;&#x2013;HDFS&#x6216;&#x8005;&#x4EFB;&#x4F55;&#x5176;&#x4ED6;Hadoop&#x652F;&#x6301;&#x7684;&#x6587;&#x4EF6;&#x7CFB;&#x7EDF;&#x3002;&#x5BF9;&#x4E8E;&#x6BCF;&#x4E2A;&#x5143;&#x7D20;&#xFF0C;Spark&#x5C06;&#x4F1A;&#x8C03;&#x7528;toString&#x65B9;&#x6CD5;&#xFF0C;&#x5C06;&#x5B83;&#x8F6C;&#x6362;&#x4E3A;&#x6587;&#x4EF6;&#x4E2D;&#x7684;&#x6587;&#x672C;&#x884C;</strong></td>
<td><strong>&#x4FDD;&#x5B58;&#x6587;&#x4EF6;&#xFF08;TXT&#x683C;&#x5F0F;&#xFF09;</strong></td>
</tr>
<tr>
<td><strong>saveAsSequenceFile(path)</strong></td>
<td><strong>&#x5C06;&#x6570;&#x636E;&#x96C6;&#x4E2D;&#x7684;&#x5143;&#x7D20;&#x4EE5;Hadoop sequencefile&#x7684;&#x683C;&#x5F0F;&#x4FDD;&#x5B58;&#x5230;&#x6307;&#x5B9A;&#x7684;&#x76EE;&#x5F55;&#x4E0B;&#xFF0C;&#x53EF;&#x4EE5;&#x662F;&#x672C;&#x5730;&#x7CFB;&#x7EDF;&#x3001;HDFS&#x6216;&#x8005;&#x4EFB;&#x4F55;&#x5176;&#x4ED6;&#x7684;Hadoop&#x652F;&#x6301;&#x7684;&#x6587;&#x4EF6;&#x7CFB;&#x7EDF;&#x3002;&#x8FD9;&#x4E2A;&#x53EA;&#x9650;&#x4E8E;&#x7531;key-value&#x5BF9;&#x7EC4;&#x6210;&#xFF0C;&#x5E76;&#x5B9E;&#x73B0;&#x4E86;Hadoop&#x7684;Writable&#x63A5;&#x53E3;&#xFF0C;&#x6216;&#x8005;&#x53EF;&#x4EE5;&#x9690;&#x5F0F;&#x7684;&#x8F6C;&#x6362;&#x4E3A;Writable&#x7684;RDD(Spark&#x5305;&#x62EC;&#x4E86;&#x57FA;&#x672C;&#x7C7B;&#x578B;&#x8F6C;&#x6362;&#xFF0C;&#x4F8B;&#x5982;Int&#x3001;Double&#x3001;String&#x7B49;)</strong></td>
<td><strong>&#x4FDD;&#x5B58;&#x6587;&#x4EF6;&#xFF08;Sequence&#x683C;&#x5F0F;&#xFF09;</strong></td>
</tr>
<tr>
<td><strong>countByKey()</strong></td>
<td><strong>&#x5BF9;(K,V)&#x7C7B;&#x578B;&#x7684;RDD&#x6709;&#x6548;&#xFF0C;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;(K,Int)&#x5BF9;&#x7684;map&#xFF0C;&#x8868;&#x793A;&#x6BCF;&#x4E00;&#x4E2A;key&#x5BF9;&#x5E94;&#x7684;&#x5143;&#x7D20;&#x4E2A;&#x6570;</strong></td>
<td><strong>&#x6570;key</strong></td>
</tr>
<tr>
<td><strong>foreach(func)</strong></td>
<td><strong>&#x5728;&#x6570;&#x636E;&#x96C6;&#x7684;&#x6BCF;&#x4E00;&#x4E2A;&#x5143;&#x7D20;&#x4E0A;&#xFF0C;&#x8FD0;&#x884C;&#x51FD;&#x6570;func&#x8FDB;&#x884C;&#x66F4;&#x65B0;&#x3002;&#x901A;&#x5E38;&#x7528;&#x4E8E;&#x8FB9;&#x7F18;&#x6548;&#x679C;&#xFF0C;&#x4F8B;&#x5982;&#x66F4;&#x65B0;&#x4E00;&#x4E2A;&#x53E0;&#x52A0;&#x5668;&#xFF0C;&#x6216;&#x8005;&#x548C;&#x5916;&#x90E8;&#x5B58;&#x50A8;&#x7CFB;&#x7EDF;&#x8FDB;&#x884C;&#x4EA4;&#x4E92;&#xFF0C;&#x5982;HBase</strong></td>
<td><strong>&#x5FAA;&#x73AF;</strong></td>
</tr>
</tbody></table>
</li>
</ul>
<h1 id="&#x9047;&#x5230;&#x7684;&#x5927;&#x5751;&#xFF01;"><a href="#&#x9047;&#x5230;&#x7684;&#x5927;&#x5751;&#xFF01;" class="headerlink" title="&#x9047;&#x5230;&#x7684;&#x5927;&#x5751;&#xFF01;"></a>&#x9047;&#x5230;&#x7684;&#x5927;&#x5751;&#xFF01;</h1><p>org.apache.spark.SparkException: Task not serializable</p>
<p>Caused by: java.io.NotSerializableException:</p>
<p>&#x89E3;&#x51B3;&#x65B9;&#x6848;&#x6211;&#x53E6;&#x5916;&#x5199;&#x4E86;&#x4E00;&#x7BC7;BLOG</p>
<p><a href="https://blog.csdn.net/HuHui_/article/details/83905302" target="_blank" rel="noopener">https://blog.csdn.net/HuHui_/article/details/83905302</a></p>
<h1 id="&#x88C5;&#x5907;"><a href="#&#x88C5;&#x5907;" class="headerlink" title="&#x88C5;&#x5907;"></a>&#x88C5;&#x5907;</h1><ol>
<li>maven&#x4F9D;&#x8D56;&#x5F15;&#x5165;</li>
<li>&#x5207;&#x5B9E;&#x7684;&#x53BB;&#x8BD5;&#x4E00;&#x4E0B;&#x600E;&#x4E48;&#x73A9;</li>
<li>&#x51FD;&#x6570;&#x5F0F;&#x7F16;&#x7A0B;&#x4E86;&#x89E3;&#x4E00;&#x4E0B;</li>
</ol>
<h1 id="Core"><a href="#Core" class="headerlink" title="Core"></a>Core</h1><p><strong>&#x5206;&#x6790;&#x524D;&#x63D0;&#x6CE8;&#x610F;</strong>&#xFF1A;</p>
<p>&#x56E0;&#x4E3A;&#x592A;&#x61D2;&#x4E86;&#xFF0C;&#x7528;&#x7684;&#x662F;&#x6211;&#x5199;&#x8FEA;&#x6770;&#x65AF;&#x7279;&#x62C9;&#x6700;&#x77ED;&#x8DDD;&#x79BB;&#x7B97;&#x6CD5;&#x751F;&#x6210;&#x7684;&#x7ED3;&#x679C;&#x6570;&#x636E;&#x6587;&#x4EF6;&#xFF0C;&#x5BF9;&#x8FD9;&#x4E2A;&#x6709;&#x5174;&#x8DA3;&#x7684;&#x770B;&#xFF1A;<a href="https://ithuhui.com/2018/10/12/algorithm-dijkstra/">&#x8FEA;&#x6770;&#x65AF;&#x7279;&#x62C9;&#xFF08;dijkstra&#xFF09;-&#x4E24;&#x4E2A;&#x5730;&#x94C1;&#x7AD9;&#x6700;&#x77ED;&#x8DDD;&#x79BB;</a></p>
<p>&#x6570;&#x636E;&#x683C;&#x5F0F;&#xFF1A;&#x8FDB;&#x7AD9;&#x7AD9;&#x70B9;&#x540D;&#x5B57;&#xFF0C;&#x51FA;&#x7AD9;&#x7AD9;&#x70B9;&#x540D;&#x5B57;&#xFF0C;&#x6700;&#x77ED;&#x8DDD;&#x79BB;&#x7ECF;&#x5386;&#x7AD9;&#x70B9;&#x6570;&#xFF0C;&#x6700;&#x77ED;&#x8DDD;&#x79BB;  </p>
<p>&#x4E3E;&#x4F8B;&#xFF1A; &#x5E7F;&#x5DDE;&#x5357;&#x7AD9;&#xFF0C;&#x5E7F;&#x5DDE;&#x5357;&#x7AD9;&#xFF0C;0,0</p>
<ul>
<li>init</li>
</ul>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">private</span> <span class="keyword">transient</span> SparkConf sparkConf;</span><br><span class="line"></span><br><span class="line"><span class="keyword">private</span> <span class="keyword">transient</span> JavaSparkContext sparkContext;</span><br><span class="line"></span><br><span class="line"><span class="keyword">private</span> <span class="keyword">static</span> <span class="keyword">final</span> String FILE_PATH = TransformationRDDTest.class.getClassLoader().getResource(<span class="string">&quot;demo.txt&quot;</span>)</span><br><span class="line">            .toString();</span><br><span class="line"><span class="meta">@Before</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">before</span><span class="params">()</span> <span class="keyword">throws</span> Exception </span>{</span><br><span class="line">        sparkConf = <span class="keyword">new</span> SparkConf().setMaster(<span class="string">&quot;local[4]&quot;</span>).setAppName(<span class="string">&quot;test&quot;</span>);</span><br><span class="line"><span class="comment">//        sparkConf.set(&quot;spark.serializer&quot;, &quot;org.apache.spark.serializer.JavaSerializer&quot;);</span></span><br><span class="line">        sparkContext = <span class="keyword">new</span> JavaSparkContext(sparkConf);</span><br><span class="line">    }</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">     * &#x6253;&#x5370;&#x6D4B;&#x8BD5;.</span></span><br><span class="line"><span class="comment">     *</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> collect the collect</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="function"><span class="keyword">private</span> <span class="keyword">void</span> <span class="title">checkResult</span><span class="params">(List&lt;?&gt; collect)</span> </span>{</span><br><span class="line">        <span class="keyword">for</span> (Object o : collect) {</span><br><span class="line">            System.out.println(o.toString());</span><br><span class="line">        }</span><br><span class="line">    }</span><br></pre></td></tr></table></figure>

<ul>
<li>Transformation RDD:</li>
</ul>
<ol>
<li>flatMap</li>
</ol>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">     * &#x5143;&#x7D20;&#x8F6C;&#x6362;. &#x53C2;&#x6570;-&gt;&#x6570;&#x7EC4;&#x53C2;&#x6570;</span></span><br><span class="line"><span class="comment">     * demo&#x8BA1;&#x7B97;&#x76EE;&#x7684;&#xFF1A;&#x83B7;&#x53D6;&#x5730;&#x94C1;&#x7AD9;&#x4FE1;&#x606F;&#x5207;&#x5206;&#x540E; &#x83B7;&#x53D6;&#x6570;&#x7EC4;&#x4FE1;&#x606F;1.&#x51FA;&#x53D1;&#x7AD9; 2.&#x7EC8;&#x70B9;&#x7AD9; 3.&#x7ECF;&#x5386;&#x7AD9;&#x70B9;&#x6570; 4.&#x8DDD;&#x79BB;</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Test</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testFlatMap</span><span class="params">()</span> </span>{</span><br><span class="line">        JavaRDD&lt;String&gt; textRDD = sparkContext.textFile(FILE_PATH);</span><br><span class="line">        JavaRDD&lt;String&gt; splitRDD = textRDD</span><br><span class="line">                .flatMap(x -&gt; Arrays.asList(x.split(<span class="string">&quot;,&quot;</span>)).iterator());</span><br><span class="line">        checkResult(splitRDD.collect());</span><br><span class="line">    }</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>mapPartitions</li>
</ol>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">    * &#x5143;&#x7D20;&#x8F6C;&#x6362;,&#x5728;&#x6BCF;&#x4E00;&#x4E2A;&#x5206;&#x533A;&#x5185;&#x90E8;&#x8FDB;&#x884C;&#x5143;&#x7D20;&#x8F6C;&#x6362;.</span></span><br><span class="line"><span class="comment">    * demo&#x8BA1;&#x7B97;&#x76EE;&#x7684;&#xFF1A;&#x7B97;&#x5E73;&#x65B9;&#x3002;&#xFF08;&#x5355;&#x5143;&#x6D4B;&#x8BD5;&#x6BD4;&#x8F83;&#x96BE;&#x770B;&#x51FA;&#x6765;&#x5206;&#x533A;&#x4F5C;&#x7528;&#xFF09;</span></span><br><span class="line"><span class="comment">    * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">    */</span></span><br><span class="line">   <span class="meta">@Test</span></span><br><span class="line">   <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testMapPartitions</span><span class="params">()</span> </span>{</span><br><span class="line">       JavaRDD&lt;Integer&gt; parallelize = sparkContext.parallelize(Arrays.asList(<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">10</span>), <span class="number">3</span>);</span><br><span class="line">       JavaRDD&lt;Tuple2&lt;Integer, Integer&gt;&gt; rdd = parallelize</span><br><span class="line">               .mapPartitions(x -&gt; getSquare(x));</span><br><span class="line">       checkResult(rdd.collect());</span><br><span class="line"></span><br><span class="line">   }</span><br><span class="line"></span><br><span class="line">   <span class="comment">/**</span></span><br><span class="line"><span class="comment">    * Gets square.</span></span><br><span class="line"><span class="comment">    *</span></span><br><span class="line"><span class="comment">    * <span class="doctag">@param</span> it the it</span></span><br><span class="line"><span class="comment">    * <span class="doctag">@return</span> the square</span></span><br><span class="line"><span class="comment">    * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">    */</span></span><br><span class="line">   <span class="meta">@Transient</span></span><br><span class="line">   <span class="keyword">private</span> Iterator&lt;Tuple2&lt;Integer, Integer&gt;&gt; getSquare(Iterator&lt;Integer&gt; it) {</span><br><span class="line">       ArrayList&lt;Tuple2&lt;Integer, Integer&gt;&gt; results = <span class="keyword">new</span> ArrayList&lt;&gt;();</span><br><span class="line">       <span class="keyword">while</span> (it.hasNext()) {</span><br><span class="line">           Integer next = it.next();</span><br><span class="line">           results.add(<span class="keyword">new</span> Tuple2&lt;&gt;(next, next * next));</span><br><span class="line">       }</span><br><span class="line">       <span class="keyword">return</span> results.iterator();</span><br><span class="line">   }</span><br></pre></td></tr></table></figure>

<ol start="3">
<li>mapPartitionsWithIndex</li>
</ol>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">     * &#x5143;&#x7D20;&#x8F6C;&#x6362;,&#x5728;&#x6BCF;&#x4E00;&#x4E2A;&#x5206;&#x533A;&#x5185;&#x90E8;&#x8FDB;&#x884C;&#x5143;&#x7D20;&#x8F6C;&#x6362;.</span></span><br><span class="line"><span class="comment">     * demo&#x8BA1;&#x7B97;&#x76EE;&#x7684;&#xFF1A;&#x7B97;&#x5E73;&#x65B9;&#x3002;&#xFF08;&#x53C2;&#x6570;1&#x662F;&#x5206;&#x533A;&#x7684;&#x7D22;&#x5F15;&#xFF09;</span></span><br><span class="line"><span class="comment">     *</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Test</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testMapPartitionsWithIndex</span><span class="params">()</span></span>{</span><br><span class="line">        JavaRDD&lt;Integer&gt; parallelize = sparkContext.parallelize(Arrays.asList(<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">10</span>), <span class="number">3</span>);</span><br><span class="line">        JavaRDD&lt;Tuple2&lt;Integer, Integer&gt;&gt; rdd = parallelize.mapPartitionsWithIndex((x, y) -&gt; getSquareWithIndex(x, y), <span class="keyword">false</span>);</span><br><span class="line">        checkResult(rdd.collect());</span><br><span class="line">    }</span><br><span class="line"></span><br><span class="line">    <span class="comment">/**</span></span><br><span class="line"><span class="comment">     * Get square with index iterator.</span></span><br><span class="line"><span class="comment">     *</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> partIndex the part index</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> it        the it</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@return</span> the iterator</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Transient</span></span><br><span class="line">    <span class="keyword">public</span> Iterator&lt;Tuple2&lt;Integer, Integer&gt;&gt; getSquareWithIndex(Integer partIndex, Iterator&lt;Integer&gt; it){</span><br><span class="line">        ArrayList&lt;Tuple2&lt;Integer, Integer&gt;&gt; results = <span class="keyword">new</span> ArrayList&lt;&gt;();</span><br><span class="line">        <span class="keyword">while</span> (it.hasNext()) {</span><br><span class="line">            Integer next = it.next();</span><br><span class="line">            results.add(<span class="keyword">new</span> Tuple2&lt;&gt;(partIndex, next * next));</span><br><span class="line">        }</span><br><span class="line">        <span class="keyword">return</span> results.iterator();</span><br><span class="line">    }</span><br></pre></td></tr></table></figure>

<ol start="4">
<li>join</li>
</ol>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">     * &#x96C6;&#x5408;&#x5173;&#x8054;. &#x5408;&#x5E76;&#x76F8;&#x540C;key&#x7684;value</span></span><br><span class="line"><span class="comment">     * demo&#x8BA1;&#x7B97;&#x76EE;&#x7684;&#xFF1A;&#x4ECA;&#x5E74;&#x548C;&#x53BB;&#x5E74;&#x90FD;&#x83B7;&#x5956;&#x7684;&#x540C;&#x5B66;&#xFF0C;&#x83B7;&#x5956;&#x9879;&#x7684;&#x79D1;&#x76EE;&#x90FD;&#x6709;&#x54EA;&#x4E9B;</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Test</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testJoin</span><span class="params">()</span> </span>{</span><br><span class="line">        <span class="comment">//&#x4ECA;&#x5E74;&#x540C;&#x5B66;&#x83B7;&#x5956;&#x7684;&#x79D1;&#x76EE;</span></span><br><span class="line">        JavaPairRDD&lt;Object, Object&gt; rdd1 = sparkContext.parallelize(Arrays.asList(</span><br><span class="line">                <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaoming&quot;</span>, <span class="string">&quot;&#x8BED;&#x6587;&quot;</span>)</span><br><span class="line">                , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaoming&quot;</span>, <span class="string">&quot;&#x6570;&#x5B66;&quot;</span>)</span><br><span class="line">                , <span class="keyword">new</span> Tuple2(<span class="string">&quot;lihua&quot;</span>, <span class="string">&quot;&#x6570;&#x5B66;&quot;</span>)</span><br><span class="line">                , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaofeng&quot;</span>, <span class="string">&quot;&#x827A;&#x672F;&quot;</span>)</span><br><span class="line">                , <span class="keyword">new</span> Tuple2(<span class="string">&quot;test&quot;</span>, <span class="string">&quot;&#x827A;&#x672F;&quot;</span>)))</span><br><span class="line">                .mapToPair(x -&gt; <span class="keyword">new</span> Tuple2&lt;&gt;(x._1, x._2));</span><br><span class="line">        <span class="comment">//&#x53BB;&#x5E74;&#x540C;&#x5B66;&#x83B7;&#x5956;&#x7684;&#x79D1;&#x76EE;</span></span><br><span class="line">        JavaPairRDD&lt;Object, Object&gt; rdd2 = sparkContext.parallelize(Arrays.asList(</span><br><span class="line">                <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaoming&quot;</span>, <span class="string">&quot;&#x827A;&#x672F;&quot;</span>)</span><br><span class="line">                , <span class="keyword">new</span> Tuple2(<span class="string">&quot;lihua&quot;</span>, <span class="string">&quot;&#x827A;&#x672F;&quot;</span>)</span><br><span class="line">                , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaofeng&quot;</span>, <span class="string">&quot;&#x8BED;&#x6587;&quot;</span>)))</span><br><span class="line">                .mapToPair(x -&gt; <span class="keyword">new</span> Tuple2&lt;&gt;(x._1, x._2));</span><br><span class="line">        JavaPairRDD&lt;Object, Tuple2&lt;Object, Object&gt;&gt; join = rdd1.join(rdd2);</span><br><span class="line">        checkResult(join.collect());</span><br><span class="line">    }</span><br></pre></td></tr></table></figure>

<ol start="5">
<li>coGroup</li>
</ol>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">    * Test co group.</span></span><br><span class="line"><span class="comment">    * demo&#x8BA1;&#x7B97;&#x76EE;&#x7684;: &#x4EE5;&#x6210;&#x7EE9;&#x5206;&#x7EC4; &#x540C;&#x5B66;([&#x6210;&#x7EE9;&#x4F18;&#x79C0;&#x5B66;&#x79D1;],[&#x6210;&#x7EE9;&#x4E2D;&#x7B49;&#x5B66;&#x79D1;],[&#x6210;&#x7EE9;&#x5DEE;&#x52B2;&#x5B66;&#x79D1;])</span></span><br><span class="line"><span class="comment">    * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">    */</span></span><br><span class="line">   <span class="meta">@Test</span></span><br><span class="line">   <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testCoGroup</span><span class="params">()</span> </span>{</span><br><span class="line">       <span class="comment">//&#x6210;&#x7EE9;&#x4F18;&#x79C0;&#x7684;&#x5B66;&#x751F;+&#x79D1;&#x76EE;</span></span><br><span class="line">       JavaRDD&lt;Tuple2&lt;String, String&gt;&gt; scoreDetails1 = sparkContext.parallelize(Arrays.asList(</span><br><span class="line">               <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaoming&quot;</span>, <span class="string">&quot;&#x8BED;&#x6587;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaoming&quot;</span>, <span class="string">&quot;&#x6570;&#x5B66;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;lihua&quot;</span>, <span class="string">&quot;&#x6570;&#x5B66;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaofeng&quot;</span>, <span class="string">&quot;&#x827A;&#x672F;&quot;</span>)));</span><br><span class="line">       <span class="comment">//&#x6210;&#x7EE9;&#x4E2D;&#x7B49;&#x7684;&#x5B66;&#x751F;+&#x79D1;&#x76EE;</span></span><br><span class="line">       JavaRDD&lt;Tuple2&lt;String, String&gt;&gt; scoreDetails2 = sparkContext.parallelize(Arrays.asList(</span><br><span class="line">               <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaoming&quot;</span>, <span class="string">&quot;&#x827A;&#x672F;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;lihua&quot;</span>, <span class="string">&quot;&#x827A;&#x672F;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaofeng&quot;</span>, <span class="string">&quot;&#x8BED;&#x6587;&quot;</span>)));</span><br><span class="line">       <span class="comment">//&#x6210;&#x7EE9;&#x5DEE;&#x7684;&#x5B66;&#x751F;+&#x79D1;&#x76EE;</span></span><br><span class="line">       JavaRDD&lt;Tuple2&lt;String, String&gt;&gt; scoreDetails3 = sparkContext.parallelize(Arrays.asList(</span><br><span class="line">               <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaoming&quot;</span>, <span class="string">&quot;&#x82F1;&#x8BED;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;lihua&quot;</span>, <span class="string">&quot;&#x82F1;&#x8BED;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;lihua&quot;</span>, <span class="string">&quot;&#x6570;&#x5B66;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaofeng&quot;</span>, <span class="string">&quot;&#x6570;&#x5B66;&quot;</span>)</span><br><span class="line">               , <span class="keyword">new</span> Tuple2(<span class="string">&quot;xiaofeng&quot;</span>, <span class="string">&quot;&#x82F1;&#x8BED;&quot;</span>)));</span><br><span class="line"></span><br><span class="line">       JavaPairRDD&lt;String, String&gt; scoreMapRDD1 = JavaPairRDD.fromJavaRDD(scoreDetails1);</span><br><span class="line">       JavaPairRDD&lt;String, String&gt; scoreMapRDD2 = JavaPairRDD.fromJavaRDD(scoreDetails2);</span><br><span class="line">       JavaPairRDD&lt;String, String&gt; scoreMapRDD3 = JavaPairRDD.fromJavaRDD(scoreDetails3);</span><br><span class="line"></span><br><span class="line">       JavaPairRDD&lt;String, Tuple3&lt;Iterable&lt;String&gt;, Iterable&lt;String&gt;, Iterable&lt;String&gt;&gt;&gt; cogroupRDD =</span><br><span class="line">               scoreMapRDD1.cogroup(scoreMapRDD2, scoreMapRDD3);</span><br><span class="line">       checkResult(cogroupRDD.collect());</span><br><span class="line">   }</span><br></pre></td></tr></table></figure>

<ul>
<li>Action RDD:</li>
</ul>
<ol>
<li>reduce</li>
</ol>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment">     * &#x805A;&#x5408;&#xFF08;&#x6574;&#x5408;&#x6570;&#x636E;&#xFF09;.</span></span><br><span class="line"><span class="comment">     *</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Test</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">testReduce</span><span class="params">()</span> </span>{</span><br><span class="line">        JavaRDD&lt;Integer&gt; parallelize = sparkContext.parallelize(Arrays.asList(<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>), <span class="number">3</span>);</span><br><span class="line">        Tuple2&lt;Integer, Integer&gt; reduce = parallelize.mapToPair(x -&gt; <span class="keyword">new</span> Tuple2&lt;&gt;(x, <span class="number">1</span>))</span><br><span class="line">                .reduce((x, y) -&gt; getReduce(x, y));</span><br><span class="line">        System.out.println(<span class="string">&quot;&#x6570;&#x7EC4;sum:&quot;</span> + reduce._1 + <span class="string">&quot; &#x8BA1;&#x7B97;&#x6B21;&#x6570;:&quot;</span> + (reduce._2 - <span class="number">1</span>));</span><br><span class="line">    }</span><br><span class="line"></span><br><span class="line">    <span class="comment">/**</span></span><br><span class="line"><span class="comment">     * &#x8BA1;&#x7B97;&#x903B;&#x8F91;.</span></span><br><span class="line"><span class="comment">     * &#xFF08;x&#xFF09;&#x603B;&#x548C;-&gt;&#x6570;&#x7EC4;&#x7684;&#x6BCF;&#x4E00;&#x4E2A;&#x6570;&#x76F8;&#x52A0;&#x603B;&#x548C;</span></span><br><span class="line"><span class="comment">     * &#xFF08;y&#xFF09;&#x603B;&#x548C; -&gt;&#x8BA1;&#x7B97;&#x6B21;&#x6570;</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> x the x</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@param</span> y the y</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@return</span> the reduce</span></span><br><span class="line"><span class="comment">     * <span class="doctag">@since</span> hui_project 1.0.0</span></span><br><span class="line"><span class="comment">     */</span></span><br><span class="line">    <span class="meta">@Transient</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> Tuple2 <span class="title">getReduce</span><span class="params">(Tuple2&lt;Integer, Integer&gt; x, Tuple2&lt;Integer, Integer&gt; y)</span> </span>{</span><br><span class="line">        Integer a = x._1();</span><br><span class="line">        Integer b = x._2();</span><br><span class="line">        a += y._1();</span><br><span class="line">        b += y._2();</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">new</span> Tuple2(a, b);</span><br><span class="line">    }</span><br></pre></td></tr></table></figure>

<h1 id="Github"><a href="#Github" class="headerlink" title="Github"></a>Github</h1><h1 id="Github-1"><a href="#Github-1" class="headerlink" title="Github"></a>Github</h1><p><strong>&#x5730;&#x5740;&#xFF1A;<a href="https://github.com/ithuhui/hui-base-java" target="_blank" rel="noopener">https://github.com/ithuhui/hui-base-java</a><br>&#x6A21;&#x5757;&#xFF1A;&#x3010;hui-base-spark&#x3011;<br>&#x5206;&#x652F;&#xFF1A;master<br>&#x4F4D;&#x7F6E;&#xFF1A;com.hui.base.spark.rdd</strong></p>
<h1 id="&#x603B;&#x7ED3;"><a href="#&#x603B;&#x7ED3;" class="headerlink" title="&#x603B;&#x7ED3;"></a>&#x603B;&#x7ED3;</h1><ol>
<li>RDD&#x5B9E;&#x64CD;&#x51E0;&#x6B21;&#xFF0C;&#x4E2A;&#x4EBA;&#x611F;&#x89C9;&#x90FD;&#x4E0D;&#x96BE;</li>
<li>API&#x4E0D;&#x96BE;&#xFF0C;&#x4F46;&#x662F;&#x8981;&#x8003;&#x8651;&#x5206;&#x533A;&#xFF0C;&#x5185;&#x5B58;&#xFF0C;&#x6027;&#x80FD;&#x624D;&#x662F;&#x91CD;&#x70B9;</li>
<li>&#x5E94;&#x7528;&#x573A;&#x666F;&#x4E00;&#x822C;&#x5728;&#x6570;&#x636E;&#x5206;&#x6790;&#xFF0C;&#x7136;&#x540E;&#x5206;&#x6790;&#x8FC7;&#x7A0B;&#x4E2D;&#x4FDD;&#x5B58;&#x7ED3;&#x679C;&#x96C6;</li>
</ol>

    </div>

    
    
    
        
      
        

<div>
<ul class="post-copyright">
  <li class="post-copyright-author">
    <strong>本文作者： </strong>冷大锤`</li>
  <li class="post-copyright-link">
    <strong>本文链接：</strong>
    <a href="http://ithuhui.com/2018/11/10/spark-rdd-common-java-api/" title="spark-rdd-JavaAPI手册">http://ithuhui.com/2018/11/10/spark-rdd-common-java-api/</a>
  </li>
  <li class="post-copyright-license">
    <strong>版权声明： </strong>本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="noopener" target="_blank"><i class="fa fa-fw fa-creative-commons"></i>BY-NC-SA</a> 许可协议。转载请注明出处！</li>
</ul>
</div>

      

      <footer class="post-footer">
          
            
          
          <div class="post-tags">
            
              <a href="/tags/spark/" rel="tag"># spark</a>
            
          </div>
        

        

          <div class="post-nav">
            <div class="post-nav-next post-nav-item">
              
                <a href="/2018/11/10/spark-task-no-serializable/" rel="next" title="spark Task no serializable 异常解决">
                  <i class="fa fa-chevron-left"></i> spark Task no serializable 异常解决
                </a>
              
            </div>

            <span class="post-nav-divider"></span>

            <div class="post-nav-prev post-nav-item">
              
                <a href="/2018/11/11/hadoop-cluster-build/" rel="prev" title="Hadoop完全分布式集群搭建">
                  Hadoop完全分布式集群搭建 <i class="fa fa-chevron-right"></i>
                </a>
              
            </div>
          </div>
        
      </footer>
    
  </div>
  
  
  
  </article>

  </div>


          </div>
          
    
    <div class="comments" id="comments"></div>
  

        </div>
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-overview">

          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image"
      src="/images/head.png"
      alt="冷大锤`">
  <p class="site-author-name" itemprop="name">冷大锤`</p>
  <div class="site-description motion-element" itemprop="description">发牌的是上帝，出牌的是自己</div>
</div>
  <nav class="site-state motion-element">
      <div class="site-state-item site-state-posts">
        
          <a href="/archives/">
        
          <span class="site-state-item-count">34</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
    
      
      
      <div class="site-state-item site-state-categories">
        
          
            <a href="/categories/">
          
        
        
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
        <span class="site-state-item-count">7</span>
        <span class="site-state-item-name">分类</span>
        </a>
      </div>
    
      
      
      <div class="site-state-item site-state-tags">
        
          
            <a href="/tags/">
          
        
        
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
        <span class="site-state-item-count">14</span>
        <span class="site-state-item-name">标签</span>
        </a>
      </div>
    
  </nav>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://github.com/ithuhui" title="GitHub &rarr; https://github.com/ithuhui" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>GitHub</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="mailto:hu545548724@163.com" title="E-Mail &rarr; mailto:hu545548724@163.com" rel="noopener" target="_blank"><i class="fa fa-fw fa-envelope"></i>E-Mail</a>
      </span>
    
  </div>




        </div>
      </div>
      <!--noindex-->
        <div class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
            
            
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#Note"><span class="nav-number">1.</span> <span class="nav-text">Note</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#遇到的大坑！"><span class="nav-number">2.</span> <span class="nav-text">遇到的大坑！</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#装备"><span class="nav-number">3.</span> <span class="nav-text">装备</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Core"><span class="nav-number">4.</span> <span class="nav-text">Core</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Github"><span class="nav-number">5.</span> <span class="nav-text">Github</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Github-1"><span class="nav-number">6.</span> <span class="nav-text">Github</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#总结"><span class="nav-number">7.</span> <span class="nav-text">总结</span></a></li></ol></div>
            

          </div>
        </div>
      <!--/noindex-->
      
        <div class="back-to-top motion-element">
          <i class="fa fa-arrow-up"></i>
          
            <span id="scrollpercent"><span>0</span>%</span>
          
        </div>
      

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; 2016 – <span itemprop="copyrightYear">2019</span>
  <span class="with-love" id="animate">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">冷大锤`</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-area-chart"></i>
    </span>
      <span class="post-meta-item-text">站点总字数：</span>
    
    <span title="站点总字数">238k</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-coffee"></i>
    </span>
      <span class="post-meta-item-text">站点阅读时长 &asymp;</span>
    
    <span title="站点阅读时长">3:36</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io" class="theme-link" rel="noopener" target="_blank">Hexo</a> 强力驱动 v3.9.0</div>
  <span class="post-meta-divider">|</span>
  <div class="theme-info">主题 – <a href="https://theme-next.org" class="theme-link" rel="noopener" target="_blank">NexT.Gemini</a> v7.3.0</div>

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item-icon">
      <i class="fa fa-user"></i>
    </span>
    <span class="site-uv" title="总访客量">
      <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
    </span>
  
    <span class="post-meta-divider">|</span>
  
    <span class="post-meta-item-icon">
      <i class="fa fa-eye"></i>
    </span>
    <span class="site-pv" title="总访问量">
      <span class="busuanzi-value" id="busuanzi_value_site_pv"></span>
    </span>
  
</div>








        
      </div>
    </footer>

    

  </div>

  
  <script src="/lib/jquery/index.js?v=3.4.1"></script>
  <script src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  <script src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

<script src="/js/utils.js?v=7.3.0"></script><script src="/js/motion.js?v=7.3.0"></script>

<script src="/js/schemes/pisces.js?v=7.3.0"></script>



<script src="/js/next-boot.js?v=7.3.0"></script>




  





  
  <script>
  function leancloudSelector(url) {
    return document.getElementById(url).querySelector('.leancloud-visitors-count');
  }
  if (CONFIG.page.isPost) {
    function addCount(Counter) {
      var visitors = document.querySelector('.leancloud_visitors');
      var url = visitors.getAttribute('id').trim();
      var title = visitors.getAttribute('data-flag-title').trim();

      Counter('get', `/classes/Counter?where=${JSON.stringify({ url })}`)
        .then(response => response.json())
        .then(({ results }) => {
          if (results.length > 0) {
            var counter = results[0];
            Counter('put', '/classes/Counter/' + counter.objectId, { time: { '__op': 'Increment', 'amount': 1 } })
              .then(response => response.json())
              .then(() => {
                leancloudSelector(url).innerText = counter.time + 1;
              })
            
              .catch(error => {
                console.log('Failed to save visitor count', error);
              })
          } else {
              Counter('post', '/classes/Counter', { title: title, url: url, time: 1 })
                .then(response => response.json())
                .then(() => {
                  leancloudSelector(url).innerText = 1;
                })
                .catch(error => {
                  console.log('Failed to create', error);
                });
            
          }
        })
        .catch(error => {
          console.log('LeanCloud Counter Error', error);
        });
    }
  } else {
    function showTime(Counter) {
      var visitors = document.querySelectorAll('.leancloud_visitors');
      var entries = [...visitors].map(element => {
        return element.getAttribute('id').trim();
      });

      Counter('get', `/classes/Counter?where=${JSON.stringify({ url: { '$in': entries } })}`)
        .then(response => response.json())
        .then(({ results }) => {
          if (results.length === 0) {
            document.querySelectorAll('.leancloud_visitors .leancloud-visitors-count').forEach(element => {
              element.innerText = 0;
            });
            return;
          }
          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.url;
            var time = item.time;
            leancloudSelector(url).innerText = time;
          }
          for (var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = leancloudSelector(url);
            if (element.innerText == '') {
              element.innerText = 0;
            }
          }
        })
        .catch(error => {
          console.log('LeanCloud Counter Error', error);
        });
    }
  }

  fetch('https://app-router.leancloud.cn/2/route?appId=u7lzh9sG1gDGC8h3Ag4jEhxD-gzGzoHsz')
    .then(response => response.json())
    .then(({ api_server }) => {
      var Counter = (method, url, data) => {
        return fetch(`https://${api_server}/1.1${url}`, {
          method: method,
          headers: {
            'X-LC-Id': 'u7lzh9sG1gDGC8h3Ag4jEhxD-gzGzoHsz',
            'X-LC-Key': 'jRXlISNU0c9YNOrnTr60TQls',
            'Content-Type': 'application/json',
          },
          body: JSON.stringify(data)
        });
      };
      if (CONFIG.page.isPost) {
        const localhost = /http:\/\/(localhost|127.0.0.1|0.0.0.0)/;
        if (localhost.test(document.URL)) return;
        addCount(Counter);
      } else if (document.querySelectorAll('.post-title-link').length >= 1) {
        showTime(Counter);
      }
    });
  </script>










  <script src="/js/local-search.js?v=7.3.0"></script>














  

  

  


  
  <script src="/js/scrollspy.js?v=7.3.0"></script><script src="/js/post-details.js?v=7.3.0"></script>



<script>
NexT.utils.getScript('//unpkg.com/valine/dist/Valine.min.js', () => {
  var GUEST = ['nick', 'mail', 'link'];
  var guest = 'nick,mail';
  guest = guest.split(',').filter(item => {
    return GUEST.indexOf(item) > -1;
  });
  new Valine({
    el: '#comments',
    verify: false,
    notify: false,
    appId: 'TIuoGjr6GXVsHwA9SzloOpGQ-gzGzoHsz',
    appKey: 'y6Va0n1Hs7FlD9RgvPCF70R3',
    placeholder: 'Just go go',
    avatar: 'mm',
    meta: guest,
    pageSize: '10' || 10,
    visitor: false,
    lang: '' || 'zh-cn',
    path: location.pathname
  });
}, window.Valine);
</script>

</body>
</html>
